import time
import pandas as pd
import numpy as np
import re
import openpyxl
import logging
import math
from openpyxl import load_workbook
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.action_chains import ActionChains
from urllib.parse import urljoin

#https://www.jianshu.com/p/56722a33ad07
#https://zhuanlan.zhihu.com/p/599579339


class PachongProjectSearch:
    def __init__(self):
        self.FileCompany = '爬虫数据/科技型企业信息Project.xlsx'  # 文件夹路径，替换为你的实际文件夹路径
        # 配置日志
        logging.basicConfig(filename='爬虫数据/科技型企业信息Project.log', level=logging.INFO,
                            format='%(asctime)s - %(levelname)s - %(message)s')

    def pachongBegin(self):
        #get直接返回，不再等待界面加载完成
        desired_capabilities = DesiredCapabilities.CHROME
        desired_capabilities["pageLoadStrategy"] = "none"
        # 设置谷歌驱动器的环境
        options = webdriver.ChromeOptions()
        # 设置chrome不加载图片，提高速度
        options.add_experimental_option("prefs", {"profile.managed_default_content_settings.images": 2})
        # 设置不显示窗口
        #options.add_argument('--headless')
        # 创建一个谷歌驱动器
        self.driver = webdriver.Chrome(options=options)
        # 最大化浏览器窗口
        self.driver.maximize_window()
        # 打开页面
        self.driver.get("https://user.medpeer.cn/user/index/login/?url=http%3A%2F%2Fnsfc.medpeer.cn%2Fshow%2Fscience_foundation%2Findex%3Fmedpeer%3D00945")

        #账号和密码登陆
        WebDriverWait(self.driver, 50).until(EC.presence_of_element_located((By.CSS_SELECTOR, '.imgApp'))).click()

        WebDriverWait(self.driver, 100).until(EC.presence_of_element_located((By.CSS_SELECTOR, '#userName'))).send_keys(
            "BOYUE")
        #dabin dabin123
        WebDriverWait(self.driver, 100).until(EC.presence_of_element_located((By.CSS_SELECTOR, '#userPsw'))).send_keys(
            "Hwc598310!")
        # print("输入password完成。")
        # 点击登陆
        WebDriverWait( self.driver, 50 ).until( EC.presence_of_element_located( (By.CSS_SELECTOR, '.login-btn') ) ).click()
        # print("已登陆。")
        # 点击去掉前置广告
        WebDriverWait(self.driver, 50).until(EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div[2]/div[5]/div[1]/div[2]/div/div[1]/i"))).click()
        # print("已去广告。")



    def PaperSingelCompany(self,nameList):
        #输入检索表达式，以作者单位搜索
        # theme = "西安标准工业股份有限公司"
        # 点击高级搜索
        highSearch = WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '.toAdvanceSearch'))).click()
        # ActionChains(self.driver).move_to_element(highSearch).click(highSearch).perform()
        # print("已点高级搜索。")
        time.sleep(2)
        # 点击以参与单位搜索
        mpSelectInput = WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div[3]/div[1]/div[2]/div[1]/div[1]/div[2]/input"))).click()
        #/html/body/div[1]/div[3]/div[1]/div[2]/div[1]/div[1]/div[2]/input
        # ActionChains(self.driver).move_to_element(mpSelectInput).click(mpSelectInput).perform()
        # print("已点项目名称。")
        WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.XPATH,
                                            f"//*[contains(@class, 'mp-select-scroll-li-text') and contains(text(), '参与单位')]"))).click()
        # ActionChains(self.driver).move_to_element(selectScrollText).click(selectScrollText).perform()
        # print("已点参与单位。")

        WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '.mp-input-con'))).send_keys(
            "{theme}".format(theme=nameList))


        WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, '.select-btn'))).click()
        time.sleep(6)

        ProItem=self.ProjectSingelCount()
        return ProItem

    def ProjectSingelCount(self):
        ProjectCount = WebDriverWait(self.driver, 50).until(
            EC.presence_of_element_located((By.XPATH,
                                            "/html/body/div[1]/div[5]/div[2]/div[2]/div[1]"))).text
        # By.CSS_SELECTOR, "button[aria-label='选择检索字段 所有字段']"
        ProjectCount = int(re.search(r'\d+', ProjectCount).group())
        print("ProjectCount",ProjectCount)
        ProItem = []
        ProContent = []
        if  ProjectCount>0:
            m = 0
            while m < math.ceil(ProjectCount / 10):
                if m != 0:
                    WebDriverWait(self.driver, 50).until(
                        EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div[5]/div[2]/div[3]/div[12]/button[2]"))).click()
                    print("点击翻页")
                    time.sleep(4)
                # 使用Selenium的方法找到目标<ul>元素，这里以XPath为例
                ul_element = WebDriverWait(self.driver, 50).until(
                    EC.presence_of_element_located((By.XPATH, "/html/body/div[1]/div[5]/div[2]/div[3]"))) #/html/body/div[1]/div[5]/div[2]/div[3]

                listItemAll = ul_element.find_elements(By.XPATH, ".//div[contains(@class, 'list-item')]")
                for j,listItem in  enumerate(listItemAll):
                    textClass = listItem.find_elements(By.XPATH,
                                                       ".//*[contains(@class, 'text_title') or contains(@class, 'short-text') or contains(@class, 'long-text')]")
                    ProContent.append(j+1)
                    for i,element in enumerate(textClass):
                        text = element.text.replace('\n', ' ')
                        parts = text.split(":")
                        if len(parts) > 1:
                            ProContent.append(parts[1])
                        else:
                            ProContent.append(element.text)
                    ProItem.append(ProContent)
                    ProContent=[]
                m=m+1
        ProItem=np.array(ProItem)
        print(ProItem.shape,ProItem)
        return ProItem
    def PaperSingeYearCount(self):
        #
        try:
            # 使用Selenium的方法找到目标<ul>元素，这里以XPath为例
            ul_element = WebDriverWait(self.driver, 8).until(
                        EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div[2]/div[1]/div[3]/dl[3]/dd/div/ul")))
            # 使用len()函数获取<ul>元素的子元素个数
            ul_length = len(ul_element.find_elements(By.TAG_NAME,"li"))
            # # 输出<ul>元素的子元素个数
            # print("ul元素的子元素个数为:", ul_length)
        except Exception as e:
            ul_length=0

        paperAllYear = []
        for i in range(1,ul_length+1,1):
            try:
                paperYear=WebDriverWait(self.driver, 8).until(
                    EC.presence_of_element_located((By.XPATH, "/html/body/div[5]/div[2]/div[1]/div[3]/dl[3]/dd/div/ul/li[{i}]/a".format(i=i)))).text
                paperYearCount = WebDriverWait(self.driver, 4).until(
                    EC.presence_of_element_located((By.XPATH,
                                                    "/html/body/div[5]/div[2]/div[1]/div[3]/dl[3]/dd/div/ul/li[{i}]/span".format(i=i)))).text
                paperYear = int(paperYear)
                paperYearCount=int(paperYearCount[1:-1])
                paperAllYear.append([paperYear, paperYearCount])
            except Exception as e:
                print("数据不完整--",i)
                logging.error(f"数据不完整--%s.",i)
                break
        paperAllYear=np.array(paperAllYear)
        paperAllYearFill=np.array([[year, 0] for year in range(2011, 2023)])
        for i in range(len(paperAllYearFill)):
            for j in range(len(paperAllYear)):
                if paperAllYearFill[i,0]== paperAllYear[j,0]:
                    paperAllYearFill[i, 1]=paperAllYear[j,1]
                    break
        print(paperAllYearFill[:,1])
        return paperAllYearFill[:,1]

    def PaperSingeYearWriteXlsx(self, start_column, start_row, data_matrix):
        '''
        :param filename: 写入文件名字
        :param column_letter: 要写入的列
        :param start_row: 开始的行
        :param data_list:  待写入的数据,类型为list
        :return:
        '''
        # 打开已存在的xlsx文件
        workbook = openpyxl.load_workbook(self.FileCompany)
        # 选择工作表
        sheet = workbook.active

        sheet.cell(row=start_row, column=5, value=data_matrix)

        # 保存修改后的xlsx文件
        workbook.save(self.FileCompany)
        print("已写入")
    def PaperAllCompany(self):
        dataCompany=self.Read_data()
        self.pachongBegin()
        i = 0
        previous_data = None
        while i < len(dataCompany):
            print(i,dataCompany[i, 2])
            paperAllYearFill=self.PaperSingelCompany(dataCompany[i,2])
            print(f"单位：{paperAllYearFill} 。")
            # self.PaperSingeYearWriteXlsx(i+2,paperAllYearFill)
            i=i+1

        self.pachongQuit()
    def get_list_union(self,input_list):
        """
        获取列表元素的并集及元素数量

        参数:
            input_list: 输入的列表

        返回:
            list_union: 列表元素的并集列表
            unique_count: 列表元素的唯一值数量
        """
        list_union = list(set(input_list))
        unique_count = len(list_union)

        return list_union, unique_count

    def count_consecutive_elements(self,column):
        """
        计算列表中连续相同元素的个数

        参数:
            column: 输入的列数组

        返回:
            count: 连续相同元素的个数
        """
        count = 1
        max_count = 1

        for i in range(1, len(column)):
            if column[i] == column[i - 1]:
                count += 1
                max_count = max(max_count, count)
            else:
                count = 1
                break

        return max_count

    def PaperAllCompanyFanPa(self):
        dataCompany = self.Read_data()
        # for i in range(len(dataCompany)):
        #     if math.isnan(dataCompany[i, 3]):
        #         pass
        # print("检测中断位置", i)
        i = 11529
        trycount = 0
        while i < len(dataCompany):
            j = 0
            try:
                self.pachongBegin()
                while j < 15000000:
                    yearCount = self.count_consecutive_elements(dataCompany[i:, 0])

                    nameList, nameCount = self.get_list_union(dataCompany[i:i + yearCount, 2])
                    print("-----------------------------------------------")
                    print(yearCount, nameList, nameCount, f"j={j}", f"i={i}")
                    logging.info("%s,  %s,  %s,  %s,  %s", yearCount, nameList, nameCount, j, i)
                    if yearCount > 2:
                        sumProCount = 0
                        dataW = []
                        for m in range(nameCount):
                            proCount = self.PaperSingelCompany(nameList[m])
                            dataW.append([nameList[m],proCount])
                            sumProCount=sumProCount+proCount
                        arraySumProCount = np.full((nameCount, 1), -2023)
                        arraySumProCount[0, 0] = sumProCount
                        dataw=np.array(dataW)
                        # print(dataW, '\r\n', arraySumProCount)
                        result_matrix = np.hstack((dataw, arraySumProCount))
                        print(f"{result_matrix}")
                        PachongProjectSearch.write_to_matrix('爬虫数据/科技型企业信息Project.xlsx',
                                                             '爬虫数据/科技型企业信息Project.xlsx', start_column=5,
                                                             start_row=i + 2, data_matrix=result_matrix)
                    else:
                        print("表中年份小于3")
                        logging.info("表中年份小于3")
                    j = j + 1
                    i = i + yearCount
                    trycount = 0
                    if i >= len(dataCompany):
                        break
                self.pachongQuit()
            except Exception as e:
                trycount = trycount + 1
                if trycount >= 3:
                    i = i + yearCount
                print("意外中止位置为：", i,"trycount为：",trycount)
                logging.warning(f"意外中止位置为：%s，trycount：%s",i,trycount)
    def PaperAllCompanyFanPaXiJie(self):
        dataCompany = self.Read_data()
        # for i in range(len(dataCompany)):
        #     if math.isnan(dataCompany[i, 3]):
        #         pass
        # print("检测中断位置", i)
        i = 14417
        trycount = 0
        while i < len(dataCompany):
            j = 0
            try:
                self.pachongBegin()
                while j < 15000000:
                    yearCount = self.count_consecutive_elements(dataCompany[i:, 0])
                    if dataCompany[i, 3]>0:
                        nameList, nameCount = self.get_list_union(dataCompany[i:i + yearCount, 2])
                        print(yearCount, nameList, nameCount, f"j={j}", f"i={i}")
                        logging.info("%s,  %s,  %s.%s,  %s",yearCount, nameList, nameCount, j, i)
                        sumProCount=0
                        dataW=[]
                        for m in range(nameCount):
                            proCount = self.PaperSingelCompany(nameList[m])
                            # print(proCount)
                            # if proCount!=[]:
                            if proCount.size != 0:
                                dataW.append(proCount)
                            # sumProCount=sumProCount+proCount
                        # arraySumProCount = np.full((nameCount, 1), -2023)
                        # arraySumProCount[0, 0] = sumProCount
                        # dataw=np.array(dataW)
                        # print(dataW, '\r\n', arraySumProCount)
                        # result_matrix = np.hstack((dataw, arraySumProCount))
                        # print(f"{result_matrix}")
                        result_matrix = np.concatenate(dataW)
                        PachongProjectSearch.write_to_matrix('爬虫数据/科技型企业信息Project.xlsx',
                                                             '爬虫数据/科技型企业信息Project.xlsx', start_column=8, start_row=i+2, data_matrix=result_matrix)
                    j = j + 1
                    i = i + yearCount
                    trycount = 0
                    if i >= len(dataCompany):
                        break
                self.pachongQuit()
            except Exception as e:
                trycount = trycount + 1
                if trycount >= 2:
                    i = i + yearCount
                print("意外中止位置为：", i,e)
                logging.warning(f"意外中止位置为：%s，错误原因：%s",i,e)
    def Read_data(self):
        dataFileCompany = pd.read_excel(self.FileCompany,\
                                  usecols=[0,1,3,6],\
                                        #,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17
                                  skiprows=[0], \
                                  header=None)

        dataFileCompany = np.array(dataFileCompany)
        dataFileCompany=self.convert_column_to_int(dataFileCompany,0)

        return dataFileCompany
    def convert_column_to_int(self,matrix_array,i):
        # Step 1: Extract the first column as a string array
        first_column_str = matrix_array[:, i]

        # Step 2: Convert the string array to int array
        first_column_int = first_column_str.astype(int)

        # Step 3: Replace the first column in the original array with the int array
        matrix_array[:, 0] = first_column_int

        return matrix_array
    def pachongQuit(self):
        # 关闭浏览器
        self.driver.close()
    def write_to_matrix(self, filename1,filename2, start_column, start_row, data_matrix):
        '''
        :param filename: 写入文件名字
        :param column_letter: 要写入的列
        :param start_row: 开始的行
        :param data_list:  待写入的数据,类型为list
        :return:
        '''
        # 打开已存在的xlsx文件
        workbook = openpyxl.load_workbook(filename1)
        # 选择工作表
        sheet = workbook.active

        # 循环遍历二维矩阵的每个元素，写入到指定位置
        for row_index, row in enumerate(data_matrix, start=start_row):
            for col_index, value in enumerate(row, start=start_column):
                if value!='-2023':
                    sheet.cell(row=row_index, column=col_index, value=value)
        # 保存修改后的xlsx文件
        workbook.save(filename2)
        print("写入完成")
if __name__ == "__main__":

    PachongProjectSearch=PachongProjectSearch()
    PachongProjectSearch.PaperAllCompanyFanPaXiJie()
    # # 配置日志
    # logging.basicConfig(filename='爬虫数据/科技型企业信息Project.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
    #
    # # 输出日志信息
    # logging.info("This is an info log message.")
    # logging.warning("This is a warning log message.")
    # logging.error("This is an error log message.")

